package cqut.wys.crawler;

import cqut.wys.utils.FileUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;

/**
 * 爬取16sucai网站的壁纸
 *
 * Created by wuyoushan on 2017/11/21.
 */
public class Crawler16sucai {

    public static void main(String[] args) {
        Crawler16sucai crawler16sucai = new Crawler16sucai();
        crawler16sucai.get16sucaiData();
    }

    private void get16sucaiData(){
        FileUtils.makeDir(FileUtils.picDir);

        // 连接站点
        // 测试爬去第3页和第4页的壁纸
        for (int i = 3; i < 4; i++) {
            // 用Jsoup连接站点
            Document doc = null;
            try {
                doc = Jsoup.connect("http://www.16sucai.com/tupian/gqfj/" + i + ".html").get();
            } catch (IOException e) {
                e.printStackTrace();
            }

            // 选择class为vector_listbox容器
            Elements elementClass = doc.select(".vector_listbox_pubu");

            // 在容器中选择a链接，用于进入相册
            Elements elements = elementClass.select("a[href~=/[0-9]{4}/[0-9]{2}/.*html]");

            System.out.println(elements.size());

            // 因为同样的链接存在与图片和文字上，做特殊处理
            for (int j = 0; j < elements.size() / 2; j++) {
                Element e = elements.get(2 * j);
                //取出该元素的title元素来新建文件夹
//                String filePath = FileUtils.picDir + "//" + e.attr("title");
                String filePath = FileUtils.picDir;
//                FileUtils.makeDir(filePath);
                // 然后在请求该链接
                System.out.println(e.attr("href"));
                if (e.attr("href").equals("/2017/07/129896.html")){
                    break;
                }
                Connection connection = Jsoup.connect("http://www.16sucai.com" + e.attr("href"));
                boolean flag = false;
                if (connection != null) {
                    System.out.println(connection);
                    System.out.println();
                    Document docInner = null;
                    try {
                        docInner = connection.get();
                    } catch (IOException e1) {
                        flag = true;
                    }
                    if (flag){
                        break;
                    }
                    // 取出对应图片的URL
                    Elements elementsClass = docInner.select(".endtext");
                    Elements elementsInner = elementsClass.select("img[src^=http://file]");
                    System.out.println(elementsInner.size());
                    // 下载图片
                    for (Element eInner : elementsInner) {
                        String picUrl = eInner.attr("src");
                        FileUtils.downloadPic(FileUtils.getInputStream(picUrl), picUrl.substring(picUrl.lastIndexOf("/")), filePath);
                    }
                }
            }
        }
    }
}
